{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GEN_5_CycleGAN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cxbxmxcx/GenReality/blob/master/GEN_5_CycleGAN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jcOwebqPwKux"
      },
      "source": [
        "#@title IMPORTS \n",
        "#@markdown run to import modules\n",
        "import argparse\n",
        "import os\n",
        "import numpy as np\n",
        "import math\n",
        "import itertools\n",
        "import scipy\n",
        "import sys\n",
        "import time\n",
        "import datetime\n",
        "\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision.utils import save_image\n",
        "\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import datasets\n",
        "from torch.autograd import Variable\n",
        "import torch.autograd as autograd\n",
        "from torchvision.utils import make_grid\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.pyplot import figure\n",
        "from IPython.display import clear_output"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rVBTY_4wwfbq",
        "outputId": "d334841a-8db2-4205-88b1-67ea42aefbde"
      },
      "source": [
        "#@title HYPERPARAMETERS \n",
        "dataset_name = \"maps\" #@param [\"facades\", \"cityscapes\", \"maps\"]\n",
        "class Hyperparameters(object):\n",
        "  def __init__(self, **kwargs):\n",
        "    self.__dict__.update(kwargs)\n",
        "\n",
        "datasets = { \"facades\" : { \"url\" : 'https://www.dropbox.com/s/0gnznt9m480rps7/facades.zip?dl=1',\n",
        "                          \"train_mode\" : \"train\",\n",
        "                          \"test_mode\" : \"test\"},\n",
        "            \"cityscapes\" : { \"url\" : \"https://www.dropbox.com/s/2a4v13fd45zwdg8/cityscapes.zip?dl=1\",\n",
        "                            \"train_mode\" : \"train\",\n",
        "                            \"test_mode\" : \"val\"},\n",
        "            \"maps\" : { \"url\" : \"https://www.dropbox.com/s/ut7g1r73cq5zdrn/maps.zip?dl=1\",\n",
        "                      \"train_mode\" : \"train\",\n",
        "                      \"test_mode\" : \"val\"}\n",
        "            }\n",
        "\n",
        "hp = Hyperparameters(\n",
        "    epoch=0,\n",
        "    n_epochs=200,\n",
        "    batch_size=1,    \n",
        "    dataset_name=dataset_name,\n",
        "    dataset_url=datasets[dataset_name][\"url\"],\n",
        "    dataset_train_mode=datasets[dataset_name][\"train_mode\"],\n",
        "    dataset_test_mode=datasets[dataset_name][\"test_mode\"],    \n",
        "    lr=.0002,\n",
        "    decay_epoch=100,\n",
        "    b1=.5,\n",
        "    b2=0.999,\n",
        "    n_cpu=8,\n",
        "    img_size=256,\n",
        "    channels=3,\n",
        "    n_critic=5,\n",
        "    sample_interval=100,\n",
        "    n_residual_blocks=3,\n",
        "    lambda_cyc=10.0,\n",
        "    lambda_id=5.0)\n",
        "\n",
        "img_root_folder = 'images'\n",
        "os.makedirs(img_root_folder, exist_ok=True)\n",
        "image_folder = img_root_folder + \"/%s\" % hp.dataset_name \n",
        "print(f\"Image data folders constructed {image_folder}\")\n",
        "os.makedirs(image_folder, exist_ok=True)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Image data folders constructed images/maps\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jzze0ldCx1O_",
        "cellView": "form",
        "outputId": "1e79714e-c7e5-4db9-c96c-4db9e725a968"
      },
      "source": [
        "#@title DOWNLOAD DATASET\n",
        "from io import BytesIO\n",
        "from urllib.request import urlopen\n",
        "from zipfile import ZipFile\n",
        "zipurl = hp.dataset_url\n",
        "with urlopen(zipurl) as zipresp:\n",
        "    with ZipFile(BytesIO(zipresp.read())) as zfile:        \n",
        "        zfile.extractall(img_root_folder)\n",
        "        print(f\"Downloaded & Extracted {zipurl}\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloaded & Extracted https://www.dropbox.com/s/ut7g1r73cq5zdrn/maps.zip?dl=1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yZaSsdJa_xqH"
      },
      "source": [
        "#@title IMAGE DOWNLOAD HELPERS AND FORATTERS\n",
        "\n",
        "import glob\n",
        "import random\n",
        "import os\n",
        "from torch.utils.data import Dataset\n",
        "from PIL import Image\n",
        "\n",
        "class ImageDataset(Dataset):\n",
        "    def __init__(self, root, transforms_=None, mode=\"train\"):\n",
        "        self.transform = transforms.Compose(transforms_)\n",
        "\n",
        "        self.files = sorted(glob.glob(os.path.join(root, mode) + \"/*.*\"))\n",
        "        if mode == \"train\":\n",
        "            self.files.extend(sorted(glob.glob(os.path.join(root, \"test\") + \"/*.*\")))\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "\n",
        "        img = Image.open(self.files[index % len(self.files)])\n",
        "        w, h = img.size\n",
        "        img_A = img.crop((0, 0, w / 2, h))\n",
        "        img_B = img.crop((w / 2, 0, w, h))\n",
        "\n",
        "        if np.random.random() < 0.5:\n",
        "            img_A = Image.fromarray(np.array(img_A)[:, ::-1, :], \"RGB\")\n",
        "            img_B = Image.fromarray(np.array(img_B)[:, ::-1, :], \"RGB\")\n",
        "\n",
        "        img_A = self.transform(img_A)\n",
        "        img_B = self.transform(img_B)\n",
        "\n",
        "        return {\"A\": img_A, \"B\": img_B}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.files)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VWcpYFFnWAbv",
        "cellView": "form"
      },
      "source": [
        "#@title IMAGE VISUALIZER HELPERS\n",
        "def imshow(img,size=10):\n",
        "  img = img / 2 + 0.5     \n",
        "  npimg = img.numpy()\n",
        "  plt.figure(figsize=(size, size))\n",
        "  plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
        "  plt.show()\n",
        "\n",
        "def to_img(x):    \n",
        "    x = x.view(x.size(0)*2, hp.channels, hp.img_size, hp.img_size)\n",
        "    return x\n",
        "\n",
        "import matplotlib.image as mpimg\n",
        "def visualise_output(path, x, y):\n",
        "    img = mpimg.imread(path)\n",
        "    plt.figure(figsize=(x,y))\n",
        "    plt.imshow(img)  \n",
        "    plt.show()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pR5LjEFQARFm",
        "cellView": "form"
      },
      "source": [
        "#@title CONFIGURE DATALOADERS\n",
        "transforms_ = [\n",
        "    transforms.Resize((hp.img_size, hp.img_size), Image.BICUBIC),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
        "]\n",
        "\n",
        "dataloader = DataLoader(\n",
        "    ImageDataset(image_folder, mode=hp.dataset_train_mode, transforms_=transforms_),\n",
        "    batch_size=hp.batch_size,\n",
        "    shuffle=True,\n",
        "    num_workers=1,\n",
        ")\n",
        "val_dataloader = DataLoader(\n",
        "    ImageDataset(image_folder, mode=hp.dataset_test_mode, transforms_=transforms_),\n",
        "    batch_size=16,\n",
        "    shuffle=True,\n",
        "    num_workers=1,\n",
        ")"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "id": "HbxCwP0zXyBQ",
        "outputId": "0f9ab309-b95b-46b2-9c1a-85087ca1742b"
      },
      "source": [
        "#@title VISUALING SAMPLE DATA { run: \"auto\" }\n",
        "pic_size = 2 #@param {type:\"integer\"} {type:\"slider\", min:1, max:30, step:1}\n",
        "\n",
        "dataiter = iter(dataloader)\n",
        "images = dataiter.next()\n",
        "\n",
        "for i in range(len(images[\"A\"])):\n",
        "  imshow(make_grid([images[\"A\"][i],images[\"B\"][i]]), size=pic_size)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJcAAABbCAYAAAB+tRe2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy92a9l93Xn9/kNezjzueeONbKqWKQoihpoihI12u6WLMtO204jHaQNJI1GA+6HbiAJkodG/oI85SEvARrIQwcI0Gig03HbUdtty1I7aluDJZEUqaI4FFlz1Z3PuIfflIffPqdKFIukZNGqJFzArbr3nLP3Pmfv71m/tb7ru9YWIQTet/ftvTD5i34D79v/d+19cL1v75m9D6737T2z98H1vr1n9j643rf3zN4H1/v2ntl7Ai4hxK8LIX4khHhVCPHP3otjvG8PvomfN88lhFDAy8AXgevAd4C/H0L44c/1QO/bA2/vhef6BPBqCOFyCKEG/iXw2+/Bcd63B9zeC3CdAq7d8/f15rH37f9npn9RBxZC/B7wewBKqae0VgQhIMDdhToA4ie3bZ6RQqweEAg8geAD9+5h9evypVISvCdusXxJeNPhxL2bxGdDQEjxpvcTt0uUQkjwziGERAiJlBIhBEop1jeG6DShsh0CMn6j3/yxfvJj/uJseTriB6eTWoK3wL1vM362+aLglVde2Q8hbL55N+8FuG4AZ+75+3Tz2I9ZCOGfA/8cYDQahZ0T69TWIXWO9yCkIAiBtTXOO5aQCiEghCRVCgJoLQlSYZ3FhwAh4JwDwHv/E28uSRKUUtR1jVKKEALWeqSMgHDONccQpGlKCPGY3nuEEDjnEEKgtUYphfeWQTvn1IkNXrlymZbMWF9fp9cbUlUVSqX8t//9P2R4RoN7nOuTkxQmQwiJkiClQMp4vKXd+/svwpZxuLNQW8dTJ45IzT4+xC+l1opev09VVfzghUt88Uu/eeWt9vNeLIvfAR4RQpwXQqTAfwH827fbQAgYDAYoCcEUFNMpvioRtiJTgm7eJlWKNEnI0xSlwLgaFxy19VR1FS86rC48jddYXqglYJxzIMQKSEsLPhD83dcJIfDe471Ha43WGu/96nlrLc45vPfU1qJ1CsBkMidVGaaqANBKc3L7A2R+hFav8sGNSzy+8TpbrX1EqKhMoLJgbTx+fK93L/AvwoQQCARCQhCCcRW/kEtr93qUVUWSpug0ve9+fu7gCiFY4J8CfwxcAv5VCOHFd9rO1I5Oq4dUAucqEh2wdYGrFxTTI4Qz6CBQOkEIRQgKF8CFeIGdc1hr8T56IaUk+IBArLwdgA8BYw1Zlq2WLiEEiIAPFiHEylMtwVdV1Qpoy8dDCBhjAFhUlnlR0261gUC5KNBKUVUVQkjytMP5Ex8jY41xdYNa/YBB/z9ybvMvOTf8IX19gHWW2oQIsgZYcbn+xYAsEBAihh4HZYpMknsCCUjzjCTL3nYf70nMFUL4CvCVd/t67z27+3tYa1BKMBx22RgNqGqLloEQ4M7hhPHBhO5ghJASpTXeO7yPAFiCYnnhpZRopXDBE5CAJIT42uA8xhjSNKWqKu5ewICUApA453/Mey09lpRyBeD4PCgVqKuSzeGQUDiMtRhjIcBkMmYy3qXfSzl74gluHmqOJzfxyhL0ETo9Ziu7zMic4nB2nrHp4YNCK5ASQhMO/CKWSiECSsKiVCA0IBDBU5cleaeNrSrKorzv9g8EQy+lpNfr4Hy8INZ65osCpRVOKJCaYb/H9saQ+XhMPS/QUkATBy3joqU55+JP8IgGBCEEtNQoJMEHrLUopRp3L1Y/UiqWu1oue8v3mCQJaZrGZZe7sZEPgaq29Lt9+oMOg7UBxlmuXr3C3t4uWiXgPYvpmFOjD7C1cR4hE4LVUGt8FVD+GjuD73J+eBXhLcaCd2L1Xv6ml0nRJDZSCGqfMLM5SEEQUJUl1bzg+OAQ13jvt7IHAlxCSB55+CLroxEqkSwWc4yJSxo2ZipKeJSWID3T6RGTwyNE41Hgbsa39DDNjlcAEiI+p6RCNmAry5I8z5FyeRoEzt0F09KWy+QyzlJKkSYx1tBak2jNfFEidUpZ1xjnuHbtFtNZwTKrSrMM7yzTyRGjzjke2voY/e4GKheQW0JqcKYgkZc4P7pEKgoqF1Zec/k+/uasyZgFBCk4KLNV3BV8oFgsMM5i3f3B9QujIu410VAJm+tb1N4wm0xYzAvq2tHttsFb+t0e1hmEVBwcHlGVNdOxod3pIBOFX+2r8SaNx7HWIqUkTZMYk+HBR5Dhoa4tnU6LoihwLnrARGvwAUPAB7/a5zI+iyCVpErjnCEEySJUWBcISrK3v09RVUipSNMUHzzBx+29cywmR+gk46HhR7GhprQzivqYWbnPolwQ/FXOjyzXxxeZ1j0SDUotvclb0zPvlUkBSsDRIuFcO8EaAyIggkArjRT3908PBLicc9y4fo2Hzp3jIx/7OJ12lz/+93/InTt3KI1nfHzMbFazsblFlkq2NjQHRwcs5hWL+ZwkS0hbrVX4G2OjuzHYEhxKKYL3qDTBWY/zHpzDe0WaZlRVvQripVbI4BFBQQj4e+gIIQQ+eKQQqCCx1hFUYO/gmH5nwMRM6HRyslShlMBZR1EWxJhOk6UpdVUwqwqkVmiVsJY/xHr3AsbPOTy6xuH8FmeGJbvTx9gr+yRBEVfjJcv33lMWQggCASkDc6spyRAsWFJC3jk87r7bPxDLopSKc+cfptVq862//Aa7uzc5sbPDYx94nI8+8REeOns2LnsicHB0zHiyoNsbMFzvIlVgMZ0xPTomEZJEJ0B03UsgLJczgCRNI0iUhIaaWHJeK+4qxEVWKoVScsWBwd0l0nmP9R60ROr4uus3riGDINGa9dEao7URGxubcTkLkR323mOsI4gEmWQoleBsjO0W0xlh4Tix9ihn1p/AmynrvW+x3b6CdxZrxd/o0rg8lpIAihf21jjyA5yPVJBRM6ya33f7BwJcSknOnn2IO7duspjP+ea3/pIQBJ/+9GdZFDOSRHHixDbOWlCS8WwCPpAmbdI8Q6ca5wLzyZRcKvIkQUsZFw/vkYgVheCsJdEaKUDrJoYIUFU1rVZrxY2tiETvCeLuib53iQQwxuGDoFosGLTazGcztFIsZnNGozXW19eaikBjDcC0jkuKs5Z2tw/BI4XDWsN0ckwnG3J66wkwkkHnRc4OrhG8wVr+xmMwIQRaCQqbcnk8ZG4yrCwpsj28rO+73QMBrtoY/vzPv4apSoZrG2RZi8uXX+dP/uSP2N/bo98b0Ol06A37VHWFEILpYs7+/hH9bpdev4NKBHVVYsoSKQRplpLphDRJ7/JZAZSI2WKAlbdaeqNlgH8vYbj0WPcG+FJKrI3lkECgms0YtjukSmHqepX9vvzaK7xy+bW7TJUQkXjUOm5pDe3ukLquqcoZUmu8ALxjMR2T6y5nNj8MNqWVv8SF/n7k81z4G8kiVxwgIGUg1QYVLK8vRkzUAodtqJK3tgcCXMF70rzDYLTBfD7j7OkzOG+5c+cO3nvmRcHLl1/l0kuXqKqCLI2AUEpga8twuMZw2CXPE47HY2xV44xDKhVriSGglI4MvXf4ENBaR5KwAc2ShLXWkuc5SdIsr032uYy3lrYEmCkKNgY92lncX7vdxnrP/tEYlWSkabKqHCRJuor7nPO0+mvU9RxTzMEFzKJAON/EjbCYjcmTDmfXP4IzmqT9Iqe6BzjbeNT3GmCBWFMVILFkoUQrQUWPW+MPMylONBziW9sDAS6lNRsbG3zmc7/KmYfO8K3vfJPdwz3G0wl5q83h8YTBYA2dKFp5hlSKTqvN5sYmQmtaacaJrR12TmzR6qTMplOcMQjiBU2SFO9ijOQBT2TXQwhoLVdLoXMOYwze+xWntfRuSZKsPKAM8Qthq5qttSGZkoAgb3fwQnD78IjCOKTSZGneLCuxhGJNjQ+ebq9PVS6wdYXSGpXohk6JpReVJMhEs5hNaaVdzqw/QV1Zup3nGeT72IYHW9rPH2AxPRKA9DXSVwSh8apLojXGb7B3/BST4sR99/BAgEsrzYkTJ/na17/O9599HpCsDfqcPnWCT3/qMzz6yCNUVcHO1gk2hxvkScbOzgk+9cynOLV9kul0Sp5mpHmLLM8IwTE5PsaUFcF7tNS00pxWlhN8wPt4Eb2PTH2epWRpuvJQdV3/WHF6mQwsC90+gLOGU5vrDDotpFb0en2Ms+wdjlEixYXAojIsaotQCqkldVXiQqDdG1AUC0xVIlSKJ1ImCFBJAkJQVxUiBBSwmE3J0wFnhh/C2pph/69I1B7GO96iNv9zsyACYJGhROCwqkUQEikFiRYolWDN8L7bPxDgyvKMeTnl5dde4ujwCGcdwQFecvXqG7zy8iVmsznr6xuc3N5ha31EMZvy/HPPcevWTZxzHB4e4q2j1cpJUgUEDg4OqMqKQCCIuLy18hZK6ZgFCoH04K0lTZMVoeobD5amKUmiV0to8KHJMCtObY5IZWT610frVM5xa/+QwjosHiEVUkpCAK0TjDWgJCppU8zm2LpEJ3HJdNY2K1BcvpM01vGCB5Ek5J0uZTGl0x5ysvdBUmfZHjyH8AXW/fyXxyWlI4NF+hJPipV9QLCMDKQUaCWbctlb2wMBrmKx4Ob1K+S5ZmNjjbyVsbG+ze7eLq+/cRlrLb1Om9s3bzBfLNBaM5lOuHb9Ot5brPF02j0ODg5RQtHt90haGvA4azB1TVnXFFVFUVUxU6MJ8LVCCol1FimjJEcrhbWWqqrI85w8T8nzDGRAesdDO9vIELA+0O32WJQVN+7sU1v/Y5mmUpKAp65LpNB0eyOEtzhbkySx6OucXVEVQkiqqmgK4hFcebtLCJ4QHIvpMcPeJpudD5BSsN5/EWct3nKPBuuvDzAByGCQfk5A4FUWtXbL55dBvgooef/jPRDgCiHQbXU5f/Ycw8GAX/783+Lzn/88g36fWzdvUZUFidacPnWaX3rqacaLGQtXo6RGJyngOXf+Ap94+lPM5yXra5u08g6nTm0DBh8sBB8rAULgjMXZ+I231lLUJd57lJAkKnJlksiPlUVJqlO0SmgnmrPbG6i4I4bDIYuq5MbuHk4IRCPjWZZsvI/iQa0Unf6A+fwYawuSRDfxXY1ztilrSqSKnyd4j1SS7mBIVRZU5YIsSZFCMpsesz48xUb+MIPkNr32G1Te494skvzZrwZg0L5ACE2QbQgSEeDNlYHmNNzXHghwgWAymVEsKj7+8U/inePPv/ENrt28SVEWWOvptPss5iW379yg122zNdoABbNiig0VV65dYVEUrK2NeP3KVapFTaIzBr0u9XyOr+rVqVFSrvRZQgiUkKsg3dQ1Wks67Xb0aCYqHFTwjLodvLX44On3+8yLgtuHx7jAXa/XUBuxSqBQUpK1e8ynx4TakOgU5wPWmgaEMYSXSqN0grEWqVJ6gzWqcoEppgRrMKYmSVNEECymR2xvnGMtOcNG6xUyuYuz4P1d7dpPa8varAiWxE4QQWBFlyBkA/43A+vNqtyftAcCXN4H2p0uD1+4yIsvvsALLzzHlWtX0FlCb9Dn/Nkz/NoXfo3BoM8bV69RFjXWVnzwscc4feI0yIRXX3+dV199lbyVM1wf4kPg9q3btFpddKKoygUYg4RVgL5UOMgl9xUCWZbhnKOqKrImY/S2Zq2T46xFak2326Osa3aPxxjvEVKgpfoxrVekNqKCdj4d451FpxnOh0gGN5mYkBKpk1hJMAYtJb3BgHIxwzuDUFH4GMFokUrgfaCYTji59SjDZJuN3iVcKLEOQvhpS0JhBSwZLMoXBKkxqh3f4dviNKok7mcPBLgg8PnP/Qp5K+fO7i2uXLtCnqZcPHeB3/rNv0Owlq9+9Y957fVXKRYLsqzFxQuP8/RTn+Czn/kVTmzvoFPB8fiAYrHgH/+jf8wnn/kE21sbZHlGp9+n081JE0VLa9I0Wcl0YmFboJPoceq6JgBJqhEEEgknRgOCd+RZRivPCULS6gxI0hYIiQ8BF+5qvOJHinVL6ywiiOixmgrBsvistCZNEhIdC+BCCDr9EYv5DO8dIUQqZWl1XeO9J89yhPeUxZyzJz7IUEpG7ddwzuNdLDO9WYZ0/zMPghCB5RZAghV9vFAgwjs4p7ejUB+QwnUIgW9/55vcvHmDo+MjvPVsrI04feoMW1vbbJ48yXMvPE+epBhb8+FPPIOUCik1+wd3OPfQQ7R7HUSAv/tbf483Xn8NVy3Y2d7m6vXbHE8mnNzZYDwZY+oKneUEFRs1tFIIIVcCQyHit9EaQyZhNBxEj5IkSCHQOqHy4IXg4vkL3Lh5g8PJMc75VR2SEMCHpoEkNOpXi3eOpSuQ90inTVU1HiynnE0IwTefT6zUrELezSaddzgh6bRb1HbOma3HaO3eQfoF+1WXTASkehceLDTeE4d2BU5qgojB+7v2f2+D3wcCXFmW8eILL1KZiiRP6Xa7tFspTzzxBN/77rcRwNmzZ9nf2+PihQ/w9NOfBODrX/8at27fYHJ8xIee+BjGGJ577jkuX34F5wzXb95mPJtiipLJVMfYSQaEEqg0JUlbeBel0T7cldeEEMi0ZHtjnWKxIE1ThJSUdU1d1mxubOGbGuHFiw9z+/Ydrt68EZMC1cQoQTTpvEApuQKWAJCsgn9rLTrRtDs9FrM5Ab8qP0W+LZCkGVJpqqrC2BotU9r9PrenLzMubnJ29BTro21cuMncnKVyKSkg1N3M9d7qwqohSsQYS4QFXiQEkeN/qsVMvK1neyDA5b3n9OkzXLl2lV6nx+c+8zm+/e2/4P/6yh/QyjLKoiRJEz74gcd59NHH+Y/f+AZpprlx8xpnz5zj0mTCD55/jo3tbZ5/7nmSNMUFy6JYUJsqqikOjpAiivykAmtrZEdFPjwqCVfvJdeSnfURdVXR7nRw3rMoSiprsd5z8/Ztdra26fV7VKbm5IkTGGu5tXc7Bt5aI3Us8wQCqmH4jakRUpJmCSCpqhKdpnS6fYrpBIJDJSneBayrItCEvutZBaAUrV6PW+OXOCivIgJcP3qOc+tPMxp0OO9u8tL0DMZJEilXitIfE1EuhZV4pC+RKKzMQQjET6UXe/sM9YGIuax1PP74h3jqlz5OVVTM5rPIr3jH7d1dDseHeB8IQXD1yutYu+Dq1df58q//Bt1enyee+AilKXnx5R+A9BTVAhNqklSRpWlUgQpFXRuyVoZznmpRYqoqJt73dPJoATubG5RFEbMzKZkvSvaPp8yKGmcdVVVy+cob3Lx9k367i5SSC+fOcfbEqajbbyQ5sml/q+sapTU60fjgMcZSVSUqTel0BsynY1zw6DSL9U1XNyWoFKkEwVpcVSGRdPoDbo5f4LC8tmqxrP2Cq4ffo9XrsjnQXGjfBu+aZo83n+1l8G5I7AKExqp2kw2+cwb4k/aAk6jOWV69/DKLRcFHP/okP/rRS3Q6HcbjQ5w3PP3xZ/jVv/VF9vd2mUyOqeqKL3zhy9y8dZtOv8NLl19hVs9QiUSmCpkKklShc0Wn32Jjo0+rndDqtMjyhH6/Q5ZFwV5RzHHO4n0glZLTO9tUZUm73UYnKZPZjOP5HOsdZVVSW4dQiizLmE2mXHrphxwfH6GV5BNPPcVHP/TEqmRkrEVohZCCsq7QuqkvOkeStui0B8xmY3wAnSYYswR5IM1aOA9IjXEOIQV5r8P1oxc4Km/FDM9D8FFnX5ox1+88T2/UZ2dQc763i/fmx7uJQiAgVsw7QsYYC/mzUWSBtwDvXXsgwIWA2WzGyZM7fOADj5JozXhyhNYpjzz8Afr9NWbzGcPhAKUFv/z5L3BwcMDJkzv86df+iGvXXsfVjmpmKeclvvYoA1mQtKVm0JSElFSMj6aMj2dsbW1FdYWtEM5iFjO2RyOqsiRJU9IsZzKds380pq7NSmBYWUNRlVSmAgHzquSNq1d4/sUXuHHzJhfOX2CrEQgupTzd7hAlBc5EVWyed8jbHeazIySBVCc4Y5sMMdIh1lpSrbFVidaKrNfl2tFzHFW3wIEoNKFOCCY2eYgyZVYdc+Pohww3N9lsz3m4e4h3AWMjCERY0g0lAoWTLQISEcLbs6H3vW5vn00+EDEXwHQyZTo+5qWjA9IkJc/afOLpZzh9+gzPPvt9ymJGmqV8/KlP8trly1y4cJ6vfu3PKIoFOlGUsxJTW1xtcYCrDVJLuq34zZyNF3gbIqcV4ODwmFYr0gNlUfD4Y4+hpaS20O/3uX1nl8PJDC8kQkYAVM5AAGcdAkElDFIIkixyYD986RJXrl7leDK+2+pmLcVizqlTJ5lOD9FJRp53mE4O0CpFiOi5rbFIIdFJ1PonSsWYLMvJWjnXDp9lVh8gnAIrIbMgQ6OhiMV0USumi13uqJfZ3n4Uf+sGoLg8XceiSJRB+xInE4LMIMR+zZ9Zk/8OnNoDAS4lFYlWHB4dkKYpp08/xHBtyKUfPo+UkslkTPCOsw89zHyxYGNjg3/zB/+GNy6/QRAeFzxOBFSmUYminBdROlxHhWme16gkweNwJmZox0fHFIWm3Wpz8cI5tBIsFnM21jeZTKcUtSM2r0e5873iQCFFI1WWaBGTgABopZjMpqsmj2Bd9Eq2olhMGQw38M5SLqYkOqGua5JEY0yFVgqEWtEhtalJ0wzVTnnj8LsU5jhmoEYhM0OQMRNdMVUiEBJPMJqjxTW0TNncfghx6zZuILgyHhCo8FIRyH8+bR7vQFk8EOAC6PUHKwFeUS7Yzrb5tS/9Bn/wh/8WaxxrwxHz+YLaGK7dusr121cJOsqMhYxyGELAiQBSIFGAQyUaGzxISBKFA4QEreNp2d7cpJXnVFXF+vo6B4cHHE9nWBHHBzjrVsASjbYeH2mLunZYIaIoMbYwkSZpkxhIDE3TbogZY6S/BHUjtU7TlKqumthHxpqjMfjgSLIcnWveOPguhT2GIMFIhAh4oyGE6DgEKOmRyuMFCB/zvf35ZbTMWNvewt65SWiXXF+McLQiHROWeeFfB2JvH6g9EODy3nN0eEzeTplMJ+wfHtPpdun2ejz55Ce4fesmu7u7vHb5NWpfc2P3Gq72UbBXV8gkNrLG8oxEpZFy8MYhE411sdk2yMBofYC1NcEJRoM1WnmLqqrp9rrs7e2xu7dPkAqvNKlOyLIM5VQUF7qAROBCBEwArPfIe9L8OGNBxqK1lDjvyTt9tM6YHO2T5HmkHhYzBIEsy1dBvDWGEDw6zRGZiB7LjhFBgFF4r0FbhLJxOQuxPOOdBBuL7UhHIDbq3pleQnQV7VYH/CFS57w+yQkI4jiNu57vZ7LA2y6NDwS4ILBzYrtJ0Q2Vs3zzO3+JdQ4tU6ytWSxmHBZHGFviqkBV1tja4p1HWAdSYEqD1oqttSGjXo+ZKTkcz7AmYCtDlmac2thgXMyZHC8AwXw2pzvoc+v2HWpXUZoanWQ4b0mbml+ikyiXrurYpk8sdiMiuJZmbZw1IYUgS1KssBHkwVOVkYytigIlJK28Q20KhIjLqRcBYwqyVgeSwNWD71O6aTw7RiFwyDzWKoOQwD3BtGoaZ42KS6dwxJEEgdsHL7O99gEG2RbJZB/XF1yZjGI/og6rWuR70aX2QGSLSimm0zkPP3yR3/o7v8Nat02aJjz/wrPcun2d/f196qog0wm+AluaWLNTAaEkIogIMiVoJylbwwFSBDo6AxdwtUEAm6M+iZZ08h55mjIZHzNdFNy4eYtr128wmcxI04y6UUZ4b6nrisViwWKxAKLyIWq+NEnzEzXvshEaeoKPhWutNWmWYsoCa2KXTJ6lFPMZPniStI0PUXjnXU3e6kLiuXLwfSo3JiAQVYIQ4NNmERIBsSx6L39Cs7ilDilA1Cl4gbcSK2p2y5fQLUW702cn32e7PcXYwN0hP++uDvlmC2KpVn1reyDAJaRie3ubnZ0dlIaLD1+kLAzeu8hrlSWtVotyUcWpNqopEqNJc43UsZ6npGJrY53aGI6LkusH+zgZSLKMnY01tkcDxtMCU9e02m2UTpnOZkil8D5wdDDm6HCMFBJTlCwHk8hGR2+txRgTC8Q+1iKVlKuu4zgqgNhEe49CQjR9j3VdY6xDSkk5GwMerfMI5ryF1TVvHHyXys/wSISLMZpPGhTcz7usdFUBn9bgA1QaIQUicVhfcfXoWbJeRp51ONvdZS2dU9vQ6MB+xutGk2zexx4IcKVJQq+b86OXLvGjl37ExUc+yMULF6nmNVma0e33OTg+YjGb42qPcwGpJO08J88TwKMQtKRGi0Dlapzw9DotTq0P2R4NyNKUyhq0ivIVKQRnzpzmzJkzjGdjnPLIVOK8wRqHVorp0RhT1VR1RW1MbMsPHqUEOon99RJBpuK8iGUMsuS4lqoLJVUz3sk2tcQEqRMW0wlaQbs3xIqaq4ffw/g5IniwAoqUuPq+i7A7xCAdLwnCI6xcbQuCyk25dvgs3WGXPEl4pL9LW5cYA82goMZ7/ZRge9B5LucdV69foyoNp06f5i+++ZecPXOaGzeuUJQFuZBxFoQEUxiSLEEJhQhwan2LW+E2MsCw3aUOFhsi266VILjA8XiKFMTmAhEz0oDg2o3r7B8cIJRAKolupaQIqrLAuUgTVGWJlwqvQBK7qaGpTTcaMB9iUCyFWClCl5ou7z21qQjBR3VDQzNorRHW4z2kWcr86Ag794TQAgukDlKLEOGdQ+4Q/xFeEJyMAx4SB5UktP0qK5y7CVeOnuOh0ccYHxzyWH+fF462sS5FizjhMDT0xrsdFfB2UHwgPFdd19zZ3eXcQ+d45plnSGTg6pXLjEZr5E2ZpVgsCD7EUZIuDlirakNtPN0sZ9jtYLBUztJKMwZ5m27eYlFWSKUZDtdIkhRjHbOq5OqtG+we7hEDY4FGsDPsk2UJdV1S1RV5q4VWEm8N7SQj01EHFhWnyWpqQwzp49oUx1DKuyMzV71/IsZrWoP3uNrQ7o3wwTM+vM3G2hl6gxG0SkLHRDSZOBPr7WzZfBKCxJvop6Q2BLmcRXZvRiiY22OuHz/PYHOdVlrx6GAX0XRy/9Qr5Dtkiy8CS8IAABXKSURBVA8EuKSQPPPMZ0nzDl//+tc4PNynrixVaXnooYf57Gc/h9KavJ2TthK8cHjhKBYLbu/usqgsk6LGeEikoq0TMq2RAdb7PbbXhmQ6qkDH0wnzYoFOE9I8Q2hJK8s5sb1NEJLJbIFOU6w17N7ZYz6fUS1mYGo6eYtExdirdpagNSJR5K0cnagV3yUbGXUcnxll1UJqXAgYWyOkoDsY4oLBVAVSSIrZlNNrT9BO1kA5SD0h9c0oy7eaMXj3keAhWElQDZEqJNJHFl8YjTBxoo8I0YtNzB43jl5gtLlDP1twcW0fgsGau0vjuxUbvh32HwhwZVnGyRMnOXvmDDdv3mT34IjXXr/M6dOn+dVf+VW2trb40he+FEclSY9OElSSICTMpgsmk4Kj4yk3bx5weDBuFJ8OT6AyDkQcLmKcYV4U1GVFlqZIrUgSjRKwu3/IeL7AAzLRpHmGSiTGGdqtnKosKOazlUa+qmuqKvZFygAqCGQjEFyW3KyxBOcJUqATiWiGo3T6I2pTY6pFMytMIIPALAxn1j5KKtsgLCGpVxi672X2AVFLJAGpfePFBMFphNUEfKw9VglUCaJUUGmmizvcnLzIaHOHUTrm0eEBBIexAb9i/t9Zj//XCuiFEGeEEF8TQvxQCPGiEOK/bh4fCSH+RAjxSvP/WvO4EEL8z82tWZ4XQvzSOx3De4+3lpdfvkTeSkE4tnc2OXXqFM//4Hm+//3v88HHHudjjz+JQjU9c/GCBcBZHykA6yEIJosFJniMjZxUbWpmiynGGZRS9FsdMpXgKku9qBkfT6htRVEUTSzl0FlC1m2R5hlFUdBut/GuxtUl3bxFrhNkM3OV1VKpV/ow1TRnxJ7HhqIQnt5ghCkXVIs53gV8iMPkvABrK4SBM8OPImXSVJqJtcQGZUslFkDwsomxAiQeLyJJIWsVt9MWmVlEp4C8RmQGkRlIHMFrxse73Jm+wmjzFMPkiEeG+wjvseatpDpvBY677+Wt7N14Lgv8dyGEx4FngH8ihHgc+GfAV0MIjwBfbf4G+DLwSPPze8D/8k4H0Fqzu3uL6eSI4CyDTo//6nf/AZ94+pMcHu7x2c99jkuXLnHxwiOsDzYRUjXeKyUIj8fiQqDVztjZ2uTwaEFZOxAJQkiSVNJq6ShF1gqUZF4VBDxCxYA6z3IQEiFjcB+Z9hCLwzIwm02i2sEZbL0gz1LyLEULuQJWlmW0Wq2fGAMQvEcg6A7WKBZzyrJEJylSqobaMAglSPKU2pTkosup3hOooCGx4CTS3YUWRJVDiLVrRBIvtPQgy4SgPOTV3asb7k5OXE5pJjWQGg6Pb7I3v8xo6wRDfczDa/uNB1vKad5hifzr8FwhhFshhO81v0+JE5pPEW+58i+al/0L4Hea338b+N9CtG8CQyHE/QcKAEVR8MYbV9BKkadtnvjwk0it+eqf/SkvvfA8v//7/5qtrW3+8A//gFaek0sdVQBakbVzVKZIEkmQkht39um2O0ipsY1nkAKGvT5CBtb7PWQSIBF0e21a3RY7WxvoTKGExHmQShNEQEqQWtLudzmxs01RzFFaMZtOKKcTtJQEBMZaalPfbVVTmjTNYulIKaRM6PSHLGZTTLUgTZpOoRjkxIF0SlFWFSpJqOo5G92zbPUuxswvdVAn4KMHC14QbCOnThoi00p82UJoB/qexollc+HqZwmxeL5Eajg4us5xcY31zR1G8oiLa4cE5zDWr7LH5vq/BUB+TuUfIcQ54EngW8B2COFW89RtYLv5/X63Z7nFfS2QpJpWu8OHP/IUu3t7/NG/+wr7e7fQScJiOuUv/uL/ZjDoE7zFeoOvY1E6SIGQikQkBKCbJPS6bZx3GFPSzjOElLTSjLMbG5ja0A8trHdooQgCWiqhU2tuizHKCVp5RjfNCMTsdNDukEjFnYMjjo+OaLVblOUMJKRpDi6QpS1cCI1Q0DfzKCJzn7c7zCfHOGPQaY73IQKryeK891hjyJvR2+32gPlszHrrIbx37M/eIGQGUavoyUyC0AafLKU2ja5LWVC+UdG8M5Ww1POHxHDn8CpqPWVtY5twcIdH1gSvHI0wBBL9s5WH3jW4hBBd4F8D/00IYfJjgv8QghBvF9q95f5Wt2fpdDq0211u3LoFMs64euPqG0wmY3q9AV/64pc4ODziytXL7O7fYWNznYcvPMZ3/urbUAecizcaGLY69LotnPMcTSeUxYKLD53l8HhCCIJOGkspMkCqNEpKVEMo9Fvt6D1MTb/dod/uEHygDo6yMuyVY+amBC1i25lUHB0e0x/0aLd7UewnuGemlcDaOFV6Pj1k2MsIImBMdU+YEmmQtOmVdNbR7vQoFnOCtywmBVuDhwk+cLB4naA9rsxQugGRl4gqQcqAkx6Rup+6DB0QoALBGW7tvcHp7ZThaAMOD3h4TfDa0RCDItGBe1oNmr4D3vZo7wpcQoiECKz/PYTwfzQP3xFCnAgh3GqWvd3m8Z/69izr6+vhwoWHOXP6DN979lkOjvdxxtDrDPj0pz9HbWr29u4wnUwZdNdYGw3p5C0ePneBV159OY5JQtBttTDOM1vM46hxoTiczrmzf8D+8YSTWwPSJNYCl21dSqpGWeBpqSRKZYxhURQU1nLn8AhrPS448nZKkudYD/OyJM0zyrLCe8jzDKU1dVnihSLL01hWamqNWqUI5fEN0x9PrCRNkqYzSNJu95lPpxhTkiUZiU6YT8bs9B/GuZqD8R2UjkkLNonCwTSOghJBI4SPYPkp0LVa9BJPcBU3Js9zdvRL9AZDxHgPhoJXjwcYJImOkp93e4B3BJeIX8P/FbgUQvif7nnq3wL/APgfm/9//57H/6kQ4l8CnwTG9yyfb2khBP7Df/hqLJEEiRQaR1w2bty4yrPPfY9gYXtnh62tLa5dv8L4aJ9Wq8PZM2eYHB8iiINtpRbIJAbmznuO5vMokHMOY10zBsmvBvEKFTM9731TAxTU1rM7H3M8LpiOC7SOs7uqokaphEoZjK1JpWZtOODg4JgQPK08Q+kkNl8kklaSkrdyhJRUdUXWypsBvx4EJEkah/5KTbfXYzEdx9n6LMWCKXjL9PiQvj6FaXlm4Sai1lBqfKdCigBOI6S9Kyz9qUwsO+EIOhAquH70PGdHT9LqDthY7MEw8NrxGpbQSHXiNXsniL0bz/UZ4L8EfiCEeLZ57H8ggupfCSH+EXAF+M+b574C/AbwKrAA/uE7fjwB/d4AY2s++PhHGA6G/PEffwWVaJ578TmEhJMndxhtbHI8npBlHXZ3b2Lq23R7OamKAb4VgnlZsigWUR3q48ThVpZhjGH/aEqeJ6wPe1hnUVJSO4sLcSDc0qPMmqWrqitUKtFpgvaeyhhsXZGkGgnMZ3OcDVjvqCZTIDAYJKRS4GrDws8xpqbTGyKVpC4LdKLRQq+G7EqlaXd7zKaNNFqqeEenEKirKsp16gVp0ubs9oe4dgDTehfRrhBGR8/lQaifXva3xOFyyIiQHl8lOFVz9egHnBv9Et512BQHhKHg8vEQi0Q3SyQrPdhb2zuCK4TwDe7vB//2W7w+AP/knfZ7rwkpm0BYcfm1l/noR5/k0cee4OXXXkJrycZwkyzrkWYtXrn8PPN5iTOGdpaRyAQPuADWOWbVAqQkzTNqY6mtIWsIV2sDxliSZj5X1dwryONw3sVZV0Ji6+aEKYUSRMpCCvJWRjEvYs3QRjFgUZQgoN1qUxvDdDpDokhbKWVdsqhrysWc0ydPMhkfUNcVaZKjM4lUsVdgPp1E3azzcdZ+c7GDjyoMJTRKBqr5jLOjD/GGNMzqPaSPYJAyclziXXiTN12t5gL4RpmqCEmNcApTFlw5/j7nRk/jDg07+cHKgyEkejU29gEv/wQfKIo4lyp4zyuXX+HSK5fYvbPH5nCTdpqzubXFt//qW9T1AiEFvXbO5mgIRD37xuYmf/93f5ed7Z3YvZPnSK1RaUqNQ2SKbrfFxTOnG1AqtFIEYGFrCmfZPxozmxdMpwsOx3NCiOF+ohRJEu/CkSYpaTsh77fIehlpnqK0ojI1Xkqc8GSpQgpH3ng4V1cUxYRuf4QgxnRCJmRZi8nkCO9M7HV0NnbiEMcv1cY0N2hokaQtpJDUVc359Sfp6nVC4hAqxBpk4F2tiXfHjnjwAmFk7CKqFdQKYRvy1ivqecm1w2fpr68RpOBE+4BzvTFupQV7++M9EOBSWjMeT5jNZljruHX7Fotiyqc++Un+k9/8LYL3vPTyC9S1YT4uyaVk2OtDM9X55KkzZFnG/u5tTm/uMOj0orcREoInUYpWmnL2xDZJojDWUBuL8x7TLIUWz3DQbYaHOBItyZVEeKgWdRzob5tZD404Mc8y8lyTdWLzal1WLOYVd/YPcMaQqjjvSybxDmZ1XdLtj8habfI8Zz4fr+6M4ZxpUjGPCzWVWTRkcR4lOlLhA+R5G1dbzq5/lLbuQ2Ijz9UQpfe93EsiNM49J9QSUSs8gqA9JB60BxUzWKQHIymnM27sv8jaxibOC072jhi1Z9QOnPt/gSoi0ZrR+gY+QJq16aQdLpx6iA9/6Amquma0uYXwniTRDPo9Bt0uPjgmsylr6+t87MmnOH/+YV59+UccHR2ymM7x1qGTxmlbwMH1m3c4HE+5fH2PaVFRO493sfCboaktzBYl+EBVlEyankXrPcZG1atMYkknyzNUqsjbOUmqydspUtDo2OHg8BBnHUoSR1GqhHoxQ0lNuztgOj4iOIdAEYJYzYeIjR8GKVOSJFuNzPQ+0Or0qMo5xXyGWRjOjj5GrjqI1K4aM95pXQwAZUqQEp9bRGIRTRUiUvwucmmpQ3QqQmqZzY+4ffAj1ra28M7yaP+QrirjyPK3kVI8EOCq6oqnnvo4GxubTCbTOBvBB37/9/9PvvKVf8dofYvKOlp5wvpgAAKMd3T7Az7z6c/x/Wf/ij/503/P0fExxgb+09/5e/T6XbRWtNttsiyjqmvmZcWd8RgTorbLBE/tA4t5TbkwTKZzhFreATYgm25piDO9pLorZTaloZobfDOJMEk1rX4OTdOFUJK9vX3m83mc86AVeaePx2GqkjRv4VyNb3T3zsW4r6prlFQxLtORi7Me0laHcj7Fmoo0zXB1ias8Z0dPkqR5LEP6+yErdlqLIKDShMyAdk2eKJt4SzTVy0ZTHwBk5MDyiuPFLnsHb7C+eRJcwWNrB+hQv+3A3wdCLCgQfP9732U2m1EUBfuHe/R6HWbFlPlsl2vXr9Hv5qwPhigpMbWjlXfo9frcvnWTg4NdTp2M92g/Pjqk2+vy61/8Ml/7+teZTiZUKiB0nJElEljvJmz2BxwXc2aLknlR4EXM8LI8itWVUgglV6SkVE1NLghsbXAGvI3zsFqdSDcsZ8g7F+j2elHFWpYYU6OSHK008/EhIQQ6/TVC1sEUJYEQx5fbAqVirTJKpuMNs9Isp1jMwVcN8y/RaRbve0iXs6OPccU8i3GNuFDce2YhBBE5sDpBakuQy091j9dxIIwiuAg2VIgSnqUrFJ6jyU2UShhtneJw9yYPDzV7D/q9f3zwzOazeIMA56nKgul0jJCQtBP6/Rajfp8kUeg0pdvv8blf/hVOnz3LYjFne3MHax3nzl2k31/j63/2pyQi5Yu/+sVmqJsnT3NUEKgKMpGSSEk3ywmJIO/mZK2UVp4hPLGkpKI+S6dRhZEkmuDjzQ10mpG1MpJWikwUpjY4E+LrMx1vFRcc/WEPqeKgOCkERTFt7p7hmU+PyVs9VBa7g+qqRImUJMmRQq9GMXV6PZyrECLEljOpmlmqjizNKcs52qWc3fwwOrRj3fHNPL0IBB/nmgbtV40dAMKDrxOwurl3dZd+Z5OMAapMkQ6o43Rp3y7Zn15msrjNaHObgT5kp7O473V9IMAVfEBKxdHxEcbUnD/3MMErynlNW6aM+j1QiuPxhMFwyIULj3D92nXGx8cIqTh//iJf/vJv8uprr3LqzFlmkzEv/uC7OGf5u7/9n9HK2ggNWStne3OT0bBPXVvmk4Ic2YydFGRpAgGMdZHzaso5WRZVDq0sp5W3yHVKt9uh02qhvAQvkToy40rH4nJdGKrKMhqNCN5RzI7xdaQtdJIBgen0gCxvY51ByeaGDDpFaY2Qmk6vTzmfE5q71gYfxxFErZjBh0CSpJTzGSkdTm8+hjJthJN3pTlNEC9qjU+b1rRlQ2yAUCegDWmesNE/x0bnHGudE2yPzrO1foEO20ggpPG+ij6tuHX8GvN6wtr6JircH1wPxLLYbrf5whd/je9+9zusDYY8+oEPMh6Pee3VH9HKU0BELssHnvnkp1gUc954/Q3arRYnTp3m+PiYsiz55Cee5tXLr3PhkUchCKazOR/+8Mf40he+zMsvX0ILSafTIU0Ui/mcvOGsdpSk3U7BB+azqmHrRYy5GiVpmmVsjEZUVc18scB6R6oSijLq42UimjKSxVpPMIF2p8NwbcClH17i/2nv3FmjCqI4/jvZp6CJaFIEIsagjXUKgxYiCLKIVQqDYJMPYCcGP4FVtLDQbyBWQpqAj9oXilarCShYhYRNCPGB7j0Wc+5y2SR6xczOZZkfXLjzWPhz9+zcmVn+c4aGBi3NcqXjznYBkqbqazNQcjkME1Uq1To/Lc9Ru+321DRJ0GzSd9vl16TtzLf79rOxtcXXb+to1Z0SLQkuI60K1NwR5Kmf1qUTGUDKbQ5Uhlkpfd723WiibH5fpy0/SMc7pcWn5hojB4/Q2tg9a5mEzALfESGyCTRD68jJMLAaWkROeqX1qKqOdFcWYuQCmqo6GVpEHkTkVdSaj0LMuSL9SQyuiDeKElz3Qwv4B6LWnBRiQh/pT4oyckX6kODBJSIXRKRpPscbf/+Edz3efZp7rLckIm9EZMHKx0Tkuel5ICJVq69Zecnax31rCxpcIlIC7uK8jieBGfNEhsS7T3OPuYaz+6XcAuZV9TjQAmatfhZoWf289fNL9rifXl/AFLCYKc8BcyE17aDxEXAet8k7anWjuL05gHvATKZ/p18PtI3hAv0csIDbQl8Fyt3PF1gEpuy+bP3Ep77Qr8XdPI6F4D99mr3gNnCd9KAdOAysq+qvHbR0dFr7hvX3RujgKizdPs1sm7qff9BltohcBFZU9XVIHX8i9N8/uTyOvcaHT9MDp4FLItIA6sAgcAd3fELZRqesllTnFxEpA0PAmk+BoUeul8AJW+FUgcs432Mwcvg0YbtP86qtGk+Rw6e5F6jqnKqOqeo47rk9VdUrwDNgehedqf5p6+939C3AhLkBfACWgZsF0HMG98p7B7y1q4GbnzwBPgKPgUPWX3Ar3mXgPTAZQPNZYMHuJ4AXON/oQ6Bm9XUrL1n7hG9dcYc+4o3Qr8VIHxODK+KNGFwRb8TgingjBlfEGzG4It6IwRXxRgyuiDd+A3KWOheIQ+hRAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 144x144 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "OXqLUivNDrNa"
      },
      "source": [
        "#@title HELPER CLASSES - ReplayBuffer & LambdaLR\n",
        "class ReplayBuffer:\n",
        "    def __init__(self, max_size=50):\n",
        "        assert max_size > 0, \"Empty buffer or trying to create a black hole. Be careful.\"\n",
        "        self.max_size = max_size\n",
        "        self.data = []\n",
        "\n",
        "    def push_and_pop(self, data):\n",
        "        to_return = []\n",
        "        for element in data.data:\n",
        "            element = torch.unsqueeze(element, 0)\n",
        "            if len(self.data) < self.max_size:\n",
        "                self.data.append(element)\n",
        "                to_return.append(element)\n",
        "            else:\n",
        "                if random.uniform(0, 1) > 0.5:\n",
        "                    i = random.randint(0, self.max_size - 1)\n",
        "                    to_return.append(self.data[i].clone())\n",
        "                    self.data[i] = element\n",
        "                else:\n",
        "                    to_return.append(element)\n",
        "        return Variable(torch.cat(to_return))\n",
        "\n",
        "\n",
        "class LambdaLR:\n",
        "    def __init__(self, n_epochs, offset, decay_start_epoch):\n",
        "        assert (n_epochs - decay_start_epoch) > 0, \"Decay must start before the training session ends!\"\n",
        "        self.n_epochs = n_epochs\n",
        "        self.offset = offset\n",
        "        self.decay_start_epoch = decay_start_epoch\n",
        "\n",
        "    def step(self, epoch):\n",
        "        return 1.0 - max(0, epoch + self.offset - self.decay_start_epoch) / (self.n_epochs - self.decay_start_epoch)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EEb5TdBmIy7l",
        "cellView": "form"
      },
      "source": [
        "#@title GENERATOR & DISCRIMINATOR\n",
        "def weights_init_normal(m):\n",
        "    classname = m.__class__.__name__\n",
        "    if classname.find(\"Conv\") != -1:\n",
        "        torch.nn.init.normal_(m.weight.data, 0.0, 0.02)\n",
        "        if hasattr(m, \"bias\") and m.bias is not None:\n",
        "            torch.nn.init.constant_(m.bias.data, 0.0)\n",
        "    elif classname.find(\"BatchNorm2d\") != -1:\n",
        "        torch.nn.init.normal_(m.weight.data, 1.0, 0.02)\n",
        "        torch.nn.init.constant_(m.bias.data, 0.0)\n",
        "\n",
        "\n",
        "##############################\n",
        "#           RESNET\n",
        "##############################\n",
        "\n",
        "\n",
        "class ResidualBlock(nn.Module):\n",
        "    def __init__(self, in_features):\n",
        "        super(ResidualBlock, self).__init__()\n",
        "\n",
        "        self.block = nn.Sequential(\n",
        "            nn.ReflectionPad2d(1),\n",
        "            nn.Conv2d(in_features, in_features, 3),\n",
        "            nn.InstanceNorm2d(in_features),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.ReflectionPad2d(1),\n",
        "            nn.Conv2d(in_features, in_features, 3),\n",
        "            nn.InstanceNorm2d(in_features),\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return x + self.block(x)\n",
        "\n",
        "\n",
        "class GeneratorResNet(nn.Module):\n",
        "    def __init__(self, input_shape, num_residual_blocks):\n",
        "        super(GeneratorResNet, self).__init__()\n",
        "\n",
        "        channels = input_shape[0]\n",
        "\n",
        "        # Initial convolution block\n",
        "        out_features = 64\n",
        "        model = [\n",
        "            nn.ReflectionPad2d(channels),\n",
        "            nn.Conv2d(channels, out_features, 7),\n",
        "            nn.InstanceNorm2d(out_features),\n",
        "            nn.ReLU(inplace=True),\n",
        "        ]\n",
        "        in_features = out_features\n",
        "\n",
        "        # Downsampling\n",
        "        for _ in range(2):\n",
        "            out_features *= 2\n",
        "            model += [\n",
        "                nn.Conv2d(in_features, out_features, 3, stride=2, padding=1),\n",
        "                nn.InstanceNorm2d(out_features),\n",
        "                nn.ReLU(inplace=True),\n",
        "            ]\n",
        "            in_features = out_features\n",
        "\n",
        "        # Residual blocks\n",
        "        for _ in range(num_residual_blocks):\n",
        "            model += [ResidualBlock(out_features)]\n",
        "\n",
        "        # Upsampling\n",
        "        for _ in range(2):\n",
        "            out_features //= 2\n",
        "            model += [\n",
        "                nn.Upsample(scale_factor=2),\n",
        "                nn.Conv2d(in_features, out_features, 3, stride=1, padding=1),\n",
        "                nn.InstanceNorm2d(out_features),\n",
        "                nn.ReLU(inplace=True),\n",
        "            ]\n",
        "            in_features = out_features\n",
        "\n",
        "        # Output layer\n",
        "        model += [nn.ReflectionPad2d(channels), nn.Conv2d(out_features, channels, 7), nn.Tanh()]\n",
        "\n",
        "        self.model = nn.Sequential(*model)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.model(x)\n",
        "\n",
        "\n",
        "##############################\n",
        "#        Discriminator\n",
        "##############################\n",
        "\n",
        "\n",
        "class Discriminator(nn.Module):\n",
        "    def __init__(self, input_shape):\n",
        "        super(Discriminator, self).__init__()\n",
        "\n",
        "        channels, height, width = input_shape\n",
        "\n",
        "        # Calculate output shape of image discriminator (PatchGAN)\n",
        "        self.output_shape = (1, height // 2 ** 4, width // 2 ** 4)\n",
        "\n",
        "        def discriminator_block(in_filters, out_filters, normalize=True):\n",
        "            \"\"\"Returns downsampling layers of each discriminator block\"\"\"\n",
        "            layers = [nn.Conv2d(in_filters, out_filters, 4, stride=2, padding=1)]\n",
        "            if normalize:\n",
        "                layers.append(nn.InstanceNorm2d(out_filters))\n",
        "            layers.append(nn.LeakyReLU(0.2, inplace=True))\n",
        "            return layers\n",
        "\n",
        "        self.model = nn.Sequential(\n",
        "            *discriminator_block(channels, 64, normalize=False),\n",
        "            *discriminator_block(64, 128),\n",
        "            *discriminator_block(128, 256),\n",
        "            *discriminator_block(256, 512),\n",
        "            nn.ZeroPad2d((1, 0, 1, 0)),\n",
        "            nn.Conv2d(512, 1, 4, padding=1)\n",
        "        )\n",
        "\n",
        "    def forward(self, img):\n",
        "        return self.model(img)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YxFuX3PCKOVW",
        "outputId": "34f23257-a4f9-4d27-c826-a7c679f3a130"
      },
      "source": [
        "#@title SETUP, LOSS, INITIALIZE MODELS and BUFFERS\n",
        "\n",
        "cuda = True if torch.cuda.is_available() else False\n",
        "print(\"Using CUDA\" if cuda else \"Not using CUDA\")\n",
        "\n",
        "# Loss functions\n",
        "criterion_GAN = torch.nn.MSELoss()\n",
        "criterion_cycle = torch.nn.L1Loss()\n",
        "criterion_identity = torch.nn.L1Loss()\n",
        "\n",
        "input_shape = (hp.channels, hp.img_size, hp.img_size)\n",
        "\n",
        "# Initialize generator and discriminator\n",
        "G_AB = GeneratorResNet(input_shape, hp.n_residual_blocks)\n",
        "G_BA = GeneratorResNet(input_shape, hp.n_residual_blocks)\n",
        "D_A = Discriminator(input_shape)\n",
        "D_B = Discriminator(input_shape)\n",
        "\n",
        "if cuda:\n",
        "    G_AB = G_AB.cuda()\n",
        "    G_BA = G_BA.cuda()\n",
        "    D_A = D_A.cuda()\n",
        "    D_B = D_B.cuda()\n",
        "    criterion_GAN.cuda()\n",
        "    criterion_cycle.cuda()\n",
        "    criterion_identity.cuda()\n",
        "\n",
        "# Initialize weights\n",
        "G_AB.apply(weights_init_normal)\n",
        "G_BA.apply(weights_init_normal)\n",
        "D_A.apply(weights_init_normal)\n",
        "D_B.apply(weights_init_normal)\n",
        "\n",
        "# Buffers of previously generated samples\n",
        "fake_A_buffer = ReplayBuffer()\n",
        "fake_B_buffer = ReplayBuffer()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using CUDA\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TpEIFH7BIRwA"
      },
      "source": [
        "#@title SAMPLING IMAGES\n",
        "def sample_images(batches_done):\n",
        "    \"\"\"Saves a generated sample from the test set\"\"\"\n",
        "    imgs = next(iter(val_dataloader))\n",
        "    G_AB.eval()\n",
        "    G_BA.eval()\n",
        "    real_A = Variable(imgs[\"A\"].type(Tensor))\n",
        "    fake_B = G_AB(real_A)\n",
        "    real_B = Variable(imgs[\"B\"].type(Tensor))\n",
        "    fake_A = G_BA(real_B)\n",
        "    # Arange images along x-axis\n",
        "    real_A = make_grid(real_A, nrow=8, normalize=True)\n",
        "    real_B = make_grid(real_B, nrow=8, normalize=True)\n",
        "    fake_A = make_grid(fake_A, nrow=8, normalize=True)\n",
        "    fake_B = make_grid(fake_B, nrow=8, normalize=True)\n",
        "    # Arange images along y-axis\n",
        "    image_grid = torch.cat((real_A, fake_B, real_B, fake_A), 1)\n",
        "    path = \"images/%s/%s.png\" % (hp.dataset_name, batches_done)\n",
        "    save_image(image_grid, path, normalize=False)    \n",
        "    return path\n",
        "    "
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fgRoX2DbHSwk"
      },
      "source": [
        "#@title OPTIMIZERS\n",
        "# Optimizers\n",
        "optimizer_G = torch.optim.Adam(\n",
        "    itertools.chain(G_AB.parameters(), G_BA.parameters()), lr=hp.lr, betas=(hp.b1, hp.b2)\n",
        ")\n",
        "optimizer_D_A = torch.optim.Adam(D_A.parameters(), lr=hp.lr, betas=(hp.b1, hp.b2))\n",
        "optimizer_D_B = torch.optim.Adam(D_B.parameters(), lr=hp.lr, betas=(hp.b1, hp.b2))\n",
        "\n",
        "# Learning rate update schedulers\n",
        "lr_scheduler_G = torch.optim.lr_scheduler.LambdaLR(\n",
        "    optimizer_G, lr_lambda=LambdaLR(hp.n_epochs, hp.epoch, hp.decay_epoch).step\n",
        ")\n",
        "lr_scheduler_D_A = torch.optim.lr_scheduler.LambdaLR(\n",
        "    optimizer_D_A, lr_lambda=LambdaLR(hp.n_epochs, hp.epoch, hp.decay_epoch).step\n",
        ")\n",
        "lr_scheduler_D_B = torch.optim.lr_scheduler.LambdaLR(\n",
        "    optimizer_D_B, lr_lambda=LambdaLR(hp.n_epochs, hp.epoch, hp.decay_epoch).step\n",
        ")\n",
        "\n",
        "Tensor = torch.cuda.FloatTensor if cuda else torch.Tensor"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GiWH4SwoI7jA"
      },
      "source": [
        "#@title TRAINING\n",
        "prev_time = time.time()\n",
        "for epoch in range(hp.epoch, hp.n_epochs):\n",
        "    for i, batch in enumerate(dataloader):\n",
        "\n",
        "        # Set model input\n",
        "        real_A = Variable(batch[\"A\"].type(Tensor))\n",
        "        real_B = Variable(batch[\"B\"].type(Tensor))\n",
        "\n",
        "        # Adversarial ground truths\n",
        "        valid = Variable(Tensor(np.ones((real_A.size(0), *D_A.output_shape))), requires_grad=False)\n",
        "        fake = Variable(Tensor(np.zeros((real_A.size(0), *D_A.output_shape))), requires_grad=False)\n",
        "\n",
        "        # ------------------\n",
        "        #  Train Generators\n",
        "        # ------------------\n",
        "\n",
        "        G_AB.train()\n",
        "        G_BA.train()\n",
        "\n",
        "        optimizer_G.zero_grad()\n",
        "\n",
        "        # Identity loss\n",
        "        loss_id_A = criterion_identity(G_BA(real_A), real_A)\n",
        "        loss_id_B = criterion_identity(G_AB(real_B), real_B)\n",
        "\n",
        "        loss_identity = (loss_id_A + loss_id_B) / 2\n",
        "\n",
        "        # GAN loss\n",
        "        fake_B = G_AB(real_A)\n",
        "        loss_GAN_AB = criterion_GAN(D_B(fake_B), valid)\n",
        "        fake_A = G_BA(real_B)\n",
        "        loss_GAN_BA = criterion_GAN(D_A(fake_A), valid)\n",
        "\n",
        "        loss_GAN = (loss_GAN_AB + loss_GAN_BA) / 2\n",
        "\n",
        "        # Cycle loss\n",
        "        recov_A = G_BA(fake_B)\n",
        "        loss_cycle_A = criterion_cycle(recov_A, real_A)\n",
        "        recov_B = G_AB(fake_A)\n",
        "        loss_cycle_B = criterion_cycle(recov_B, real_B)\n",
        "\n",
        "        loss_cycle = (loss_cycle_A + loss_cycle_B) / 2\n",
        "\n",
        "        # Total loss\n",
        "        loss_G = loss_GAN + hp.lambda_cyc * loss_cycle + hp.lambda_id * loss_identity\n",
        "\n",
        "        loss_G.backward()\n",
        "        optimizer_G.step()\n",
        "\n",
        "        # -----------------------\n",
        "        #  Train Discriminator A\n",
        "        # -----------------------\n",
        "\n",
        "        optimizer_D_A.zero_grad()\n",
        "\n",
        "        # Real loss\n",
        "        loss_real = criterion_GAN(D_A(real_A), valid)\n",
        "        # Fake loss (on batch of previously generated samples)\n",
        "        fake_A_ = fake_A_buffer.push_and_pop(fake_A)\n",
        "        loss_fake = criterion_GAN(D_A(fake_A_.detach()), fake)\n",
        "        # Total loss\n",
        "        loss_D_A = (loss_real + loss_fake) / 2\n",
        "\n",
        "        loss_D_A.backward()\n",
        "        optimizer_D_A.step()\n",
        "\n",
        "        # -----------------------\n",
        "        #  Train Discriminator B\n",
        "        # -----------------------\n",
        "\n",
        "        optimizer_D_B.zero_grad()\n",
        "\n",
        "        # Real loss\n",
        "        loss_real = criterion_GAN(D_B(real_B), valid)\n",
        "        # Fake loss (on batch of previously generated samples)\n",
        "        fake_B_ = fake_B_buffer.push_and_pop(fake_B)\n",
        "        loss_fake = criterion_GAN(D_B(fake_B_.detach()), fake)\n",
        "        # Total loss\n",
        "        loss_D_B = (loss_real + loss_fake) / 2\n",
        "\n",
        "        loss_D_B.backward()\n",
        "        optimizer_D_B.step()\n",
        "\n",
        "        loss_D = (loss_D_A + loss_D_B) / 2\n",
        "\n",
        "        # --------------\n",
        "        #  Log Progress\n",
        "        # --------------\n",
        "\n",
        "        # Determine approximate time left\n",
        "        batches_done = epoch * len(dataloader) + i\n",
        "        batches_left = hp.n_epochs * len(dataloader) - batches_done\n",
        "        time_left = datetime.timedelta(seconds=batches_left * (time.time() - prev_time))\n",
        "        prev_time = time.time()\n",
        "\n",
        "        # Print log\n",
        "        sys.stdout.write(\n",
        "            \"\\r[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f, adv: %f, cycle: %f, identity: %f] ETA: %s\"\n",
        "            % (\n",
        "                epoch,\n",
        "                hp.n_epochs,\n",
        "                i,\n",
        "                len(dataloader),\n",
        "                loss_D.item(),\n",
        "                loss_G.item(),\n",
        "                loss_GAN.item(),\n",
        "                loss_cycle.item(),\n",
        "                loss_identity.item(),\n",
        "                time_left,\n",
        "            )\n",
        "        )\n",
        "\n",
        "        # If at sample interval save image\n",
        "        if batches_done % hp.sample_interval == 0:\n",
        "          clear_output()\n",
        "          visualise_output(sample_images(batches_done), 10, 10)   "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}